{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "import torchvision.datasets as datasets\n",
    "import torch.nn.functional as F\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import os\n",
    "#import matplotlib\n",
    "#import matplotlib.pyplot as plt\n",
    "from torch.optim.lr_scheduler import StepLR\n",
    "import numpy as np\n",
    "from datetime import datetime\n",
    "import pandas as pd\n",
    "import random \n",
    "from torchvision.datasets import ImageFolder\n",
    "import re\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "\n",
    "from PIL import Image\n",
    "#from torch.optim.lr_scheduler import StepLR\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from skimage.io import imread, imsave\n",
    "import skimage\n",
    "from PIL import ImageFile\n",
    "from PIL import Image\n",
    "import io\n",
    "import torchvision.transforms.functional as TF\n",
    "from torch.autograd import Variable\n",
    "import torch.nn.functional as F\n",
    "\n",
    "def resize2d(img, size):\n",
    "    return F.adaptive_avg_pool2d(Variable(img),size).data\n",
    "\n",
    "torch.cuda.empty_cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# normalize = transforms.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225])\n",
    "\n",
    "########## Mean and std are calculated from the train dataset\n",
    "normalize = transforms.Normalize(mean=[0.45271412, 0.45271412, 0.45271412],\n",
    "                                     std=[0.33165374, 0.33165374, 0.33165374])\n",
    "train_transformer = transforms.Compose([\n",
    "    transforms.Resize(256),\n",
    "    transforms.RandomResizedCrop((224),scale=(0.5,1.0)),\n",
    "    transforms.RandomHorizontalFlip(),\n",
    "#     transforms.RandomRotation(90),\n",
    "    # random brightness and random contrast\n",
    "    transforms.ColorJitter(brightness=0.2, contrast=0.2),\n",
    "    transforms.ToTensor(),\n",
    "    normalize\n",
    "])\n",
    "\n",
    "val_transformer = transforms.Compose([\n",
    "#     transforms.Resize(224),\n",
    "#     transforms.CenterCrop(224),\n",
    "    transforms.Resize((224,224)),\n",
    "    transforms.ToTensor(),\n",
    "    normalize\n",
    "])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "item,label= data/CT_NonCOVID/noncovid_ct_scan_0-53.jpg 1\n",
      "item,label= data/CT_COVID/covid_ct_scan_0-116.jpg 0\n",
      "item,label= data/CT_NonCOVID/noncovid_ct_scan_0-53.jpg 1\n",
      "item,label= data/CT_COVID/covid_ct_scan_0-116.jpg 0\n",
      "item,label= data/CT_COVID/Test_COVID/c78643e4-25.jpg 0\n",
      "item,label= data/CT_COVID/Test_COVID/0eabf7e0-2.jpg 0\n",
      "35\n",
      "35\n",
      "607\n"
     ]
    }
   ],
   "source": [
    "batchsize=4\n",
    "def read_txt(txt_path):\n",
    "    with open(txt_path) as f:\n",
    "        lines = f.readlines()\n",
    "    txt_data = [line.strip() for line in lines]\n",
    "    return txt_data\n",
    "\n",
    "class CovidCTDataset(Dataset):\n",
    "    def __init__(self, root_dir, txt_COVID, txt_NonCOVID, transform=None):\n",
    "        \"\"\"\n",
    "        Args:\n",
    "            txt_path (string): Path to the txt file with annotations.\n",
    "            root_dir (string): Directory with all the images.\n",
    "            transform (callable, optional): Optional transform to be applied\n",
    "                on a sample.\n",
    "        File structure:\n",
    "        - root_dir\n",
    "            - CT_COVID\n",
    "                - img1.png\n",
    "                - img2.png\n",
    "                - ......\n",
    "            - CT_NonCOVID\n",
    "                - img1.png\n",
    "                - img2.png\n",
    "                - ......\n",
    "        \"\"\"\n",
    "        self.root_dir = root_dir\n",
    "        self.txt_path = [txt_COVID,txt_NonCOVID]\n",
    "        self.classes = ['CT_COVID','CT_NonCOVID']\n",
    "        self.num_cls = len(self.classes)\n",
    "        self.img_list = []\n",
    "        for c in range(self.num_cls):\n",
    "            cls_list = [[os.path.join(self.root_dir,self.classes[c],item), c] for item in read_txt(self.txt_path[c])]\n",
    "            self.img_list += cls_list    \n",
    "        self.transform = transform\n",
    "        len0=len(self.img_list)\n",
    "        print('item,label=',self.img_list[len0-1][0],self.img_list[len0-1][1])\n",
    "        print('item,label=',self.img_list[1][0],self.img_list[1][1])\n",
    "        \n",
    "    def __len__(self):\n",
    "        return len(self.img_list)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        if torch.is_tensor(idx):\n",
    "            idx = idx.tolist()\n",
    "\n",
    "        img_path = self.img_list[idx][0]\n",
    "        image = Image.open(img_path).convert('RGB')\n",
    "\n",
    "        if self.transform:\n",
    "            image = self.transform(image)\n",
    "        sample = {'img': image,\n",
    "                  'label': int(self.img_list[idx][1])}\n",
    "        return sample\n",
    "\n",
    "\n",
    "\n",
    "    \n",
    "if __name__ == '__main__':\n",
    "    trainset = CovidCTDataset(root_dir='data',\n",
    "                              txt_COVID='data/trainCT-COVID.txt',\n",
    "                              txt_NonCOVID='data/trainCT-NonCOVID.txt',\n",
    "                              transform= train_transformer)\n",
    "    valset = CovidCTDataset(root_dir='data',\n",
    "                              txt_COVID='data/valCT-COVID.txt',\n",
    "                              txt_NonCOVID='data/valCT-NonCOVID.txt',\n",
    "                              transform= val_transformer)\n",
    "    testset = CovidCTDataset(root_dir='data',\n",
    "                              txt_COVID='data/test_COVID.txt',\n",
    "                              txt_NonCOVID='data/test_NonCOVID.txt',\n",
    "                              transform= val_transformer)\n",
    "    print(trainset.__len__())\n",
    "    print(valset.__len__())\n",
    "    print(testset.__len__())\n",
    "\n",
    "    train_loader = DataLoader(trainset, batch_size=batchsize, drop_last=False, shuffle=True)\n",
    "    val_loader = DataLoader(valset, batch_size=batchsize, drop_last=False, shuffle=False)\n",
    "    test_loader = DataLoader(testset, batch_size=batchsize, drop_last=False, shuffle=False)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "trainset= <__main__.CovidCTDataset object at 0x7fef0be329a0>\n",
      "img-path=,label= 0 0\n",
      "trainset= <__main__.CovidCTDataset object at 0x7fef0be329a0>\n",
      "img-path=,label= 1 0\n",
      "trainset= <__main__.CovidCTDataset object at 0x7fef0be329a0>\n",
      "img-path=,label= 2 0\n"
     ]
    }
   ],
   "source": [
    "# Random Plots\n",
    "train_no = trainset.__len__()\n",
    "random_idx = np.random.randint(1, train_no, size=3)\n",
    "#fig, axes = plt.subplots(3, 3, figsize=(16, 12))\n",
    "\n",
    "#for idx, ax in enumerate(axes.ravel()):\n",
    "for idx in range(3):\n",
    "    #img = Image.open(train_list[idx])\n",
    "    print('trainset=',trainset)\n",
    "    sample = trainset.__getitem__(idx)\n",
    "   # print('sample-shape=',sample)\n",
    "    img = sample['img']\n",
    "    label = sample['label']\n",
    "    #img_path = trainset.__getitem__(idx).img_path\n",
    "    print('img-path=,label=', idx, label)\n",
    "    #ax.set_title(labels[idx])\n",
    "   # ax.set_title(label)\n",
    "   # ax.imshow(img)\n",
    "\n",
    "\n",
    "#for batch_index, batch_samples in enumerate(train_loader):      \n",
    "    #    data, target = batch_samples['img'], batch_samples['label']\n",
    "#skimage.io.imshow(data[0,1,:,:].numpy())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#training process is defined here \n",
    "\n",
    "alpha = None\n",
    "## alpha is None if mixup is not used\n",
    "alpha_name = f'{alpha}'\n",
    "device = 'cuda'\n",
    "\n",
    "def train(optimizer, epoch):\n",
    "    \n",
    "    model.train()\n",
    "    \n",
    "    train_loss = 0\n",
    "    train_correct = 0\n",
    "    \n",
    "    for batch_index, batch_samples in enumerate(train_loader):\n",
    "        \n",
    "        # move data to device\n",
    "        data, target = batch_samples['img'].to(device), batch_samples['label'].to(device)\n",
    "        \n",
    "        ## adjust data to meet the input dimension of model\n",
    "#         data = data[:, 0, :, :]\n",
    "#         data = data[:, None, :, :]    \n",
    "        \n",
    "        #mixup\n",
    "#         data, targets_a, targets_b, lam = mixup_data(data, target, alpha, use_cuda=True)\n",
    "        \n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        output = model(data)\n",
    "        criteria = nn.CrossEntropyLoss()\n",
    "        loss = criteria(output, target.long())\n",
    "        \n",
    "        #mixup loss\n",
    "#         loss = mixup_criterion(criteria, output, targets_a, targets_b, lam)\n",
    "\n",
    "        train_loss += criteria(output, target.long())\n",
    "        \n",
    "        optimizer.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        pred = output.argmax(dim=1, keepdim=True)\n",
    "        train_correct += pred.eq(target.long().view_as(pred)).sum().item()\n",
    "    \n",
    "        # Display progress and write to tensorboard\n",
    "        if batch_index % bs == 0:\n",
    "            print('Train Epoch: {} [{}/{} ({:.0f}%)]\\tTrain Loss: {:.6f}'.format(\n",
    "                epoch, batch_index, len(train_loader),\n",
    "                100.0 * batch_index / len(train_loader), loss.item()/ bs))\n",
    "    \n",
    "#     print('\\nTrain set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\\n'.format(\n",
    "#         train_loss/len(train_loader.dataset), train_correct, len(train_loader.dataset),\n",
    "#         100.0 * train_correct / len(train_loader.dataset)))\n",
    "#     f = open('model_result/{}.txt'.format(modelname), 'a+')\n",
    "#     f.write('\\nTrain set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\\n'.format(\n",
    "#         train_loss/len(train_loader.dataset), train_correct, len(train_loader.dataset),\n",
    "#         100.0 * train_correct / len(train_loader.dataset)))\n",
    "#     f.write('\\n')\n",
    "#     f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#val process is defined here\n",
    "\n",
    "def val(epoch):\n",
    "    \n",
    "    model.eval()\n",
    "    test_loss = 0\n",
    "    correct = 0\n",
    "    results = []\n",
    "    \n",
    "    TP = 0\n",
    "    TN = 0\n",
    "    FN = 0\n",
    "    FP = 0\n",
    "    \n",
    "    \n",
    "    criteria = nn.CrossEntropyLoss()\n",
    "    # Don't update model\n",
    "    with torch.no_grad():\n",
    "        tpr_list = []\n",
    "        fpr_list = []\n",
    "        \n",
    "        predlist=[]\n",
    "        scorelist=[]\n",
    "        targetlist=[]\n",
    "        # Predict\n",
    "        for batch_index, batch_samples in enumerate(val_loader):\n",
    "            data, target = batch_samples['img'].to(device), batch_samples['label'].to(device)\n",
    "            \n",
    "#             data = data[:, 0, :, :]\n",
    "#             data = data[:, None, :, :]\n",
    "            output = model(data)\n",
    "            \n",
    "            test_loss += criteria(output, target.long())\n",
    "            score = F.softmax(output, dim=1)\n",
    "            pred = output.argmax(dim=1, keepdim=True)\n",
    "#             print('target',target.long()[:, 2].view_as(pred))\n",
    "            correct += pred.eq(target.long().view_as(pred)).sum().item()\n",
    "            \n",
    "#             print(output[:,1].cpu().numpy())\n",
    "#             print((output[:,1]+output[:,0]).cpu().numpy())\n",
    "#             predcpu=(output[:,1].cpu().numpy())/((output[:,1]+output[:,0]).cpu().numpy())\n",
    "            targetcpu=target.long().cpu().numpy()\n",
    "            predlist=np.append(predlist, pred.cpu().numpy())\n",
    "            scorelist=np.append(scorelist, score.cpu().numpy()[:,1])\n",
    "            targetlist=np.append(targetlist,targetcpu)\n",
    "           \n",
    "    return targetlist, scorelist, predlist\n",
    "    \n",
    "    # Write to tensorboard\n",
    "#     writer.add_scalar('Test Accuracy', 100.0 * correct / len(test_loader.dataset), epoch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#test process is defined here \n",
    "\n",
    "def test(epoch):\n",
    "    \n",
    "    model.eval()\n",
    "    test_loss = 0\n",
    "    correct = 0\n",
    "    results = []\n",
    "    \n",
    "    TP = 0\n",
    "    TN = 0\n",
    "    FN = 0\n",
    "    FP = 0\n",
    "    \n",
    "    \n",
    "    criteria = nn.CrossEntropyLoss()\n",
    "    # Don't update model\n",
    "    with torch.no_grad():\n",
    "        tpr_list = []\n",
    "        fpr_list = []\n",
    "        \n",
    "        predlist=[]\n",
    "        scorelist=[]\n",
    "        targetlist=[]\n",
    "        # Predict\n",
    "        for batch_index, batch_samples in enumerate(test_loader):\n",
    "            data, target = batch_samples['img'].to(device), batch_samples['label'].to(device)\n",
    "#             data = data[:, 0, :, :]\n",
    "#             data = data[:, None, :, :]\n",
    "#             print(target)\n",
    "            output = model(data)\n",
    "            \n",
    "            test_loss += criteria(output, target.long())\n",
    "            score = F.softmax(output, dim=1)\n",
    "            pred = output.argmax(dim=1, keepdim=True)\n",
    "#             print('target',target.long()[:, 2].view_as(pred))\n",
    "            correct += pred.eq(target.long().view_as(pred)).sum().item()\n",
    "#             TP += ((pred == 1) & (target.long()[:, 2].view_as(pred).data == 1)).cpu().sum()\n",
    "#             TN += ((pred == 0) & (target.long()[:, 2].view_as(pred) == 0)).cpu().sum()\n",
    "# #             # FN    predict 0 label 1\n",
    "#             FN += ((pred == 0) & (target.long()[:, 2].view_as(pred) == 1)).cpu().sum()\n",
    "# #             # FP    predict 1 label 0\n",
    "#             FP += ((pred == 1) & (target.long()[:, 2].view_as(pred) == 0)).cpu().sum()\n",
    "#             print(TP,TN,FN,FP)\n",
    "            \n",
    "            \n",
    "#             print(output[:,1].cpu().numpy())\n",
    "#             print((output[:,1]+output[:,0]).cpu().numpy())\n",
    "#             predcpu=(output[:,1].cpu().numpy())/((output[:,1]+output[:,0]).cpu().numpy())\n",
    "            targetcpu=target.long().cpu().numpy()\n",
    "            predlist=np.append(predlist, pred.cpu().numpy())\n",
    "            scorelist=np.append(scorelist, score.cpu().numpy()[:,1])\n",
    "            targetlist=np.append(targetlist,targetcpu)\n",
    "    return targetlist, scorelist, predlist\n",
    "    \n",
    "    # Write to tensorboard\n",
    "#     writer.add_scalar('Test Accuracy', 100.0 * correct / len(test_loader.dataset), epoch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"Load Self-Trans model\"\"\"\n",
    "\"\"\"Change names and locations to the Self-Trans.pt\"\"\"\n",
    "import torchvision.models as models\n",
    "\n",
    "model = models.densenet169(pretrained=True).cuda()\n",
    "# pretrained_net = torch.load('model_backup/Dense169.pt')\n",
    "# pretrained_net = torch.load('model_backup/mixup/Dense169_0.6.pt')\n",
    "#pretrained_net = torch.load('model_backup/medical_transfer/medical_transfer_None_LUNA_moco_covid_moco.pt')\n",
    "pretrained_net = torch.load('Self_Trans_xg.pt')\n",
    "\n",
    "\n",
    "model.load_state_dict(pretrained_net)\n",
    "\n",
    "modelname = 'Dense169_ssl_luna_moco'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train\n",
    "bs =batchsize\n",
    "votenum = 1 #10\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "\n",
    "r_list = []\n",
    "p_list = []\n",
    "acc_list = []\n",
    "AUC_list = []\n",
    "# TP = 0\n",
    "# TN = 0\n",
    "# FN = 0\n",
    "# FP = 0\n",
    "vote_pred = np.zeros(valset.__len__())\n",
    "vote_score = np.zeros(valset.__len__())\n",
    "\n",
    "#optimizer = optim.SGD(model.parameters(), lr=0.001, momentum = 0.9)\n",
    "optimizer = optim.Adam(model.parameters(), lr=0.0001)\n",
    "scheduler = optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=10)\n",
    "\n",
    "# scheduler = optim.lr_scheduler.ExponentialLR(optimizer, gamma = 0.95)\n",
    "                                             \n",
    "scheduler = StepLR(optimizer, step_size=1)\n",
    "#torch.save(model.state_dict(), \"model_backup/medical_transfer/{}_{}_covid_moco_covid.pt\".format(modelname,alpha_name)) \n",
    "        \n",
    "total_epoch = 0; 2 #3000; # 0=no-training\n",
    "\n",
    "for epoch in range(1, total_epoch+1):\n",
    "    train(optimizer, epoch)\n",
    "    \n",
    "    targetlist, scorelist, predlist = val(epoch)\n",
    "    print('target',targetlist)\n",
    "    print('score',scorelist)\n",
    "    print('predict',predlist)\n",
    "    vote_pred = vote_pred + predlist \n",
    "    vote_score = vote_score + scorelist \n",
    "\n",
    "    if epoch % votenum == 0:\n",
    "        \n",
    "        # major vote\n",
    "        vote_pred[vote_pred <= (votenum/2)] = 0\n",
    "        vote_pred[vote_pred > (votenum/2)] = 1\n",
    "        vote_score = vote_score/votenum\n",
    "        \n",
    "        print('vote_pred', vote_pred)\n",
    "        print('targetlist', targetlist)\n",
    "        TP = ((vote_pred == 1) & (targetlist == 1)).sum()\n",
    "        TN = ((vote_pred == 0) & (targetlist == 0)).sum()\n",
    "        FN = ((vote_pred == 0) & (targetlist == 1)).sum()\n",
    "        FP = ((vote_pred == 1) & (targetlist == 0)).sum()\n",
    "        \n",
    "        \n",
    "        print('TP=',TP,'TN=',TN,'FN=',FN,'FP=',FP)\n",
    "        print('TP+FP',TP+FP)\n",
    "        p = TP / (TP + FP)\n",
    "        print('precision',p)\n",
    "        p = TP / (TP + FP)\n",
    "        r = TP / (TP + FN)\n",
    "        print('recall',r)\n",
    "        F1 = 2 * r * p / (r + p)\n",
    "        acc = (TP + TN) / (TP + TN + FP + FN)\n",
    "        print('F1',F1)\n",
    "        print('acc',acc)\n",
    "        AUC = roc_auc_score(targetlist, vote_score)\n",
    "        print('AUCp', roc_auc_score(targetlist, vote_pred))\n",
    "        print('AUC', AUC)\n",
    "        \n",
    "        \n",
    "        \n",
    "#         if epoch == total_epoch:\n",
    "        torch.save(model.state_dict(), \"model_backup/medical_transfer/{}_{}_covid_moco_covid.pt\".format(modelname,alpha_name))  \n",
    "        torch.save(model.state_dict(), \"Self_Trans_xg.pt\")  \n",
    "        \n",
    "        vote_pred = np.zeros(valset.__len__())\n",
    "        vote_score = np.zeros(valset.__len__())\n",
    "        print('\\n The epoch is {}, average recall: {:.4f}, average precision: {:.4f},\\\n",
    "average F1: {:.4f}, average accuracy: {:.4f}, average AUC: {:.4f}'.format(\n",
    "        epoch, r, p, F1, acc, AUC))\n",
    "\n",
    "#         f = open('model_result/medical_transfer/{}_{}.txt'.format(modelname,alpha_name), 'a+')\n",
    "#         f.write('\\n The epoch is {}, average recall: {:.4f}, average precision: {:.4f},\\\n",
    "# average F1: {:.4f}, average accuracy: {:.4f}, average AUC: {:.4f}'.format(\n",
    "#         epoch, r, p, F1, acc, AUC))\n",
    "#         f.close()\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"\\nbs = 10\\nimport warnings\\nwarnings.filterwarnings('ignore')\\n\\nepoch = 1\\nr_list = []\\np_list = []\\nacc_list = []\\nAUC_list = []\\n# TP = 0\\n# TN = 0\\n# FN = 0\\n# FP = 0\\nvote_pred = np.zeros(testset.__len__())\\nvote_score = np.zeros(testset.__len__())\\ntargetlist, scorelist, predlist = test(epoch)\\nprint('target',targetlist)\\nprint('score',scorelist)\\nprint('predict',predlist)\\nvote_pred = vote_pred + predlist \\nvote_score = vote_score + scorelist \\n\\nTP = ((predlist == 1) & (targetlist == 1)).sum()\\n\\nTN = ((predlist == 0) & (targetlist == 0)).sum()\\nFN = ((predlist == 0) & (targetlist == 1)).sum()\\nFP = ((predlist == 1) & (targetlist == 0)).sum()\\n\\nprint('TP=',TP,'TN=',TN,'FN=',FN,'FP=',FP)\\nprint('TP+FP',TP+FP)\\np = TP / (TP + FP)\\nprint('precision',p)\\np = TP / (TP + FP)\\nr = TP / (TP + FN)\\nprint('recall',r)\\nF1 = 2 * r * p / (r + p)\\nacc = (TP + TN) / (TP + TN + FP + FN)\\nprint('F1',F1)\\nprint('acc',acc)\\nAUC = roc_auc_score(targetlist, vote_score)\\nprint('AUC', AUC)\\n\""
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# test\n",
    "\n",
    "\"\"\"\n",
    "bs = 10\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "epoch = 1\n",
    "r_list = []\n",
    "p_list = []\n",
    "acc_list = []\n",
    "AUC_list = []\n",
    "# TP = 0\n",
    "# TN = 0\n",
    "# FN = 0\n",
    "# FP = 0\n",
    "vote_pred = np.zeros(testset.__len__())\n",
    "vote_score = np.zeros(testset.__len__())\n",
    "targetlist, scorelist, predlist = test(epoch)\n",
    "print('target',targetlist)\n",
    "print('score',scorelist)\n",
    "print('predict',predlist)\n",
    "vote_pred = vote_pred + predlist \n",
    "vote_score = vote_score + scorelist \n",
    "\n",
    "TP = ((predlist == 1) & (targetlist == 1)).sum()\n",
    "\n",
    "TN = ((predlist == 0) & (targetlist == 0)).sum()\n",
    "FN = ((predlist == 0) & (targetlist == 1)).sum()\n",
    "FP = ((predlist == 1) & (targetlist == 0)).sum()\n",
    "\n",
    "print('TP=',TP,'TN=',TN,'FN=',FN,'FP=',FP)\n",
    "print('TP+FP',TP+FP)\n",
    "p = TP / (TP + FP)\n",
    "print('precision',p)\n",
    "p = TP / (TP + FP)\n",
    "r = TP / (TP + FN)\n",
    "print('recall',r)\n",
    "F1 = 2 * r * p / (r + p)\n",
    "acc = (TP + TN) / (TP + TN + FP + FN)\n",
    "print('F1',F1)\n",
    "print('acc',acc)\n",
    "AUC = roc_auc_score(targetlist, vote_score)\n",
    "print('AUC', AUC)\n",
    "\"\"\"\n",
    "\n",
    "# f = open(f'model_result/medical_transfer/test_{modelname}_{alpha_name}_LUNA_moco_CT_moco.txt', 'a+')\n",
    "# f.write('\\n The epoch is {}, average recall: {:.4f}, average precision: {:.4f},\\\n",
    "# average F1: {:.4f}, average accuracy: {:.4f}, average AUC: {:.4f}'.format(\n",
    "# epoch, r, p, F1, acc, AUC))\n",
    "# f.close()\n",
    "# torch.save(model.state_dict(), \"model_backup/medical_transfer/{}_{}_covid_moco_covid.pt\".format(modelname,alpha_name))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test_no= 607\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'\\ntrain_no = trainset.__len__()\\nprint(\\'train_no=\\',train_no)\\n#random_idx = np.random.randint(1, train_no, size=3)\\n#fig, axes = plt.subplots(3, 3, figsize=(16, 12))\\n#for idx, ax in enumerate(axes.ravel()):\\n\\nwith io.open(\\'train_results_Jun23.txt\\', \\'w\\') as f:\\n    for item in read_txt(\\'data/trainCT-COVID.txt\\'):\\n        #print(\\'item=\\',item)\\n        img_path = os.path.join(\\'data/CT_COVID\\',item)\\n        img = Image.open(img_path).convert(\\'RGB\\')\\n\\n    #for idx in range (10):    #range(test_no):\\n        #img = Image.open(train_list[idx])\\n       # sample = trainset.__getitem__(idx)\\n       # print(\\'sample-shape=\\',sample)\\n       # img = sample[\\'img\\']\\n        img1 = TF.to_tensor(img)\\n        img2 = resize2d(img1,(224,224))\\n        x = torch.zeros(1,3,224,224)\\n        x[0] = img2\\n        out = x[0]\\n       # out = img\\n        out.unsqueeze_(0)\\n        out = out.to(\\'cuda:0\\')\\n        model.to(\\'cuda:0\\')\\n        label = sample[\\'label\\']\\n        #img_path = trainset.__getitem__(idx).img_path  \\n        output = model(out)\\n        cls = torch.max(output[0],0)\\n        cls = cls.indices\\n        #print(\\'label, output=\\', label, cls)\\n        if cls>0:\\n            f.write(\"%s\\t%s\\n\" % (item, \\'1\\'))\\n        else:\\n            f.write(\"%s\\t%s\\n\" % (item, \\'0\\'))\\n    #ax.set_title(labels[idx])\\n   # ax.set_title(label)\\n   # ax.imshow(img)\\n'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## saving testing results\n",
    "\n",
    "\n",
    "## testing on test-dataset\n",
    "test_no = testset.__len__()\n",
    "print('test_no=',test_no)\n",
    "#random_idx = np.random.randint(1, train_no, size=3)\n",
    "#fig, axes = plt.subplots(3, 3, figsize=(16, 12))\n",
    "with io.open(f'test_DenseNet_Jun30-2.txt', 'w') as f:\n",
    "#for idx, ax in enumerate(axes.ravel()):\n",
    "\n",
    "    for item in read_txt('data/test_COVID.txt'):\n",
    "        #print('item=',item)\n",
    "        img_path = os.path.join('data',item)\n",
    "        img = Image.open(img_path).convert('RGB')\n",
    "\n",
    "    #for idx in range (10):    #range(test_no):\n",
    "        #img = Image.open(train_list[idx])\n",
    "       # sample = trainset.__getitem__(idx)\n",
    "       # print('sample-shape=',sample)\n",
    "       # img = sample['img']\n",
    "        img1 = TF.to_tensor(img)\n",
    "        img2 = resize2d(img1,(224,224))\n",
    "        x = torch.zeros(1,3,224,224)\n",
    "        x[0] = img2\n",
    "        out = x[0]\n",
    "       # out = img\n",
    "        out.unsqueeze_(0)\n",
    "        out = out.to('cuda:0')\n",
    "        model.to('cuda:0')\n",
    "        label = sample['label']\n",
    "        #img_path = trainset.__getitem__(idx).img_path  \n",
    "        output = model(out)\n",
    "        cls = torch.max(output[0],0)\n",
    "        cls = cls.indices\n",
    "        #print('label, output=', label, cls)\n",
    "        if cls>0:\n",
    "            f.write(\"%s\\t%s\\n\" % (item, '0'))\n",
    "        else:\n",
    "            f.write(\"%s\\t%s\\n\" % (item, '1'))\n",
    "        #ax.set_title(labels[idx])\n",
    "       # ax.set_title(label)\n",
    "       # ax.imshow(img)\n",
    "\n",
    "\n",
    "\n",
    "\"\"\"\n",
    "train_no = trainset.__len__()\n",
    "print('train_no=',train_no)\n",
    "#random_idx = np.random.randint(1, train_no, size=3)\n",
    "#fig, axes = plt.subplots(3, 3, figsize=(16, 12))\n",
    "#for idx, ax in enumerate(axes.ravel()):\n",
    "\n",
    "with io.open('train_results_Jun23.txt', 'w') as f:\n",
    "    for item in read_txt('data/trainCT-COVID.txt'):\n",
    "        #print('item=',item)\n",
    "        img_path = os.path.join('data/CT_COVID',item)\n",
    "        img = Image.open(img_path).convert('RGB')\n",
    "\n",
    "    #for idx in range (10):    #range(test_no):\n",
    "        #img = Image.open(train_list[idx])\n",
    "       # sample = trainset.__getitem__(idx)\n",
    "       # print('sample-shape=',sample)\n",
    "       # img = sample['img']\n",
    "        img1 = TF.to_tensor(img)\n",
    "        img2 = resize2d(img1,(224,224))\n",
    "        x = torch.zeros(1,3,224,224)\n",
    "        x[0] = img2\n",
    "        out = x[0]\n",
    "       # out = img\n",
    "        out.unsqueeze_(0)\n",
    "        out = out.to('cuda:0')\n",
    "        model.to('cuda:0')\n",
    "        label = sample['label']\n",
    "        #img_path = trainset.__getitem__(idx).img_path  \n",
    "        output = model(out)\n",
    "        cls = torch.max(output[0],0)\n",
    "        cls = cls.indices\n",
    "        #print('label, output=', label, cls)\n",
    "        if cls>0:\n",
    "            f.write(\"%s\\t%s\\n\" % (item, '1'))\n",
    "        else:\n",
    "            f.write(\"%s\\t%s\\n\" % (item, '0'))\n",
    "    #ax.set_title(labels[idx])\n",
    "   # ax.set_title(label)\n",
    "   # ax.imshow(img)\n",
    "\"\"\"\n",
    "\n",
    "\n",
    "#for batch_index, batch_samples in enumerate(train_loader):      \n",
    "#        data, target = batch_samples['img'], batch_samples['label']\n",
    "#skimage.io.imshow(data[0,1,:,:].numpy())\n",
    "# f = open(f'model_result/medical_transfer/test_{modelname}_{alpha_name}_LUNA_moco_CT_moco.txt', 'a+')\n",
    "# f.write('\\n The epoch is {}, average recall: {:.4f}, average precision: {:.4f},\\\n",
    "# average F1: {:.4f}, average accuracy: {:.4f}, average AUC: {:.4f}'.format(\n",
    "# epoch, r, p, F1, acc, AUC))\n",
    "# f.close()\n",
    "# torch.save(model.state_dict(), \"model_backup/medical_transfer/{}_{}_covid_moco_covid.pt\".format(modelname,alpha_name))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
